/*
 * CompMonitor.java
 *
 * Created on June 6, 2002, 11:01 AM
 */

package redman;

import java.lang.*;
import java.util.*;
import java.io.*;
import java.net.*;
import org.apache.log4j.*;

/**
 *
 * @author  llaranje
 * @version 
 */
public class CompMon extends Thread {
  
    String name;
    String appname;
    String type = "PROCESS";
    String startpath;
    String shutdownpath;
    String killpath;
    String dependentcomps;
 
    AppMonitor app;
    int port;
    boolean local = true;
    long threshold, timewindow, t1 = 0, t2 = 0, numfault = 0;
    boolean firsttime = true;
    boolean handlingfailure = false;
    boolean newlystarted = false;
    int subcompnum = 0;
    SubCompMon[] subcompmons = null;
    
    int numhosts = 0;
    int comppid = -1;
    String[] comphosts;
    
    boolean compdead = false;
    boolean msgsspecified = false;
    boolean managed = false;
    boolean remeventmsg = false;
    boolean loceventmsg = false;
    boolean failureHandled = false;
    boolean running = false;
    Process p = null;
    
    private static Category trace = Category.getInstance(CompMon.class);
    
    Object pgroup_sem = new Object();
    
    PrintWriter out = null;
    String host = null;
    
    TestThread[] tests = null;
    int numtests = 0;
    
    /** Creates new CompMon */
    public CompMon(String appname, AppMonitor app, String compname) {
        name = compname;
        this.appname = appname;
        this.app = app;
               
        // Read locality information (default is "local = true")
        String locality = app.appstatictable.getProperty(appname + "_" + name + "_LOCALITY");
        if (locality != null) {
            locality = locality.trim();
            if (locality.equals("DISTRIBUTED")) local = false;
        }
        
        if (((local) && (app.local)) || ((!local) && (!app.local))) running = true;
        
        if (!local && app.local) host = findDistHost();
        
        host = app.localhost;
        
        // Read type information (daemon, process or process group)
        type = app.appstatictable.getProperty(appname + "_" + name);
        if (type != null) type = type.trim();
        trace.debug("Local component " + appname + ":" + name + " type is " + type);
        
        // Read managed information
        // Default for PROCESSES and DAEMONS is "managed = false" if not specified
        // Default managed status for PROCESS_GROUPS is:  
        //      if specified all subcomponents go with the specification;
        //      if not specified, program will look for the specification at the
        //      subcomponent level (default is "managed = false")

        String stmsgs = app.appstatictable.getProperty(appname + "_" + name + "_MESSAGES");
        
        if (stmsgs != null) {
            msgsspecified = true;
            managed = true;
            if (!type.equals("PROCESS_GROUP")) app.nummanaged++;
            StringTokenizer st = new StringTokenizer(stmsgs);
            String msg;
            while (st.hasMoreTokens()) {
                msg = st.nextToken();
                if (msg.equals("REMOTE_EVENT")) remeventmsg = true;
                if (msg.equals("LOCAL_EVENT"))  loceventmsg = true;
            }
        }
        
        // Read the path of the script or executable
        startpath = app.appstatictable.getProperty(appname + "_" + name + "_START_PATH");
        if (startpath != null) startpath = startpath.trim();
        startpath = app.homedir + "/" + startpath;
        
        // Read the path of the shutdown script or executable
        shutdownpath = app.appstatictable.getProperty(appname + "_" + name + "_SHUTDOWN_PATH");
        if (shutdownpath != null) {
            shutdownpath = shutdownpath.trim();
            shutdownpath = app.homedir + "/" + shutdownpath;
        }
        
        // Read the path of the kill script or executable
        killpath = app.appstatictable.getProperty(appname + "_" + name + "_KILL_PATH");
        if (killpath != null) {
            killpath = killpath.trim();
            killpath = app.homedir + "/" + killpath;
        }
        
        // Read the list of components that depend on this component
        dependentcomps = app.appstatictable.getProperty(appname + "_" + name + "_DEPENDENTS");
        
        // Read the fault threshold for this component.
        // This is the maximum number of faults (within the fault time window)
        // that the component can undergo and be restarted without the RedMan failing,
        // cleaning up and restarting the entire application
        String stthreshold = 
            app.appstatictable.getProperty(appname + "_" + name + "_FAULT_THRESHOLD");
        if (stthreshold != null) {
            stthreshold = stthreshold.trim();             
            threshold = Integer.parseInt(stthreshold);
        }
        // trace.debug(appname + ":" + compname + " fault threshold = " + stthreshold);
        
        // Read the time threshold (to compute a failure) for this component
        String sttimewindow = 
            app.appstatictable.getProperty(appname + "_" + name + "_FAULT_TIME_WINDOW");
        if (sttimewindow != null) {
            sttimewindow = sttimewindow.trim();           
            timewindow = Integer.parseInt(sttimewindow);
        }
        // trace.debug(appname + ":" + compname + " fault time window = " + sttimewindow);
        
        // Create process group objects if this is a process group
        if (type.equals("PROCESS_GROUP")) createProcessGroupObjects();
        
        if (!running) {
            trace.info("CompMon object " + appname + ":" + compname 
                                         + " created, but app not supposed to run on this host");
        }
        else {
            trace.info("CompMon object " + appname + ":" + compname + " created");
        }
    }
    
    private String findDistHost() {
        String token = null, host = null;
        if (local) return(null);
        try {            
            String hosts = app.appstatictable.getProperty(appname + "_" + name + "_HOSTS");
            StringTokenizer st = new StringTokenizer(hosts);
       
            if ((numhosts = st.countTokens()) <= 0) {
                trace.debug(name 
                    + ": Bad config file. No host names in distributed component definition.");
                return(null);
            }
       
            int numcheck;
            if (app.numapphosts <= numhosts) numcheck = app.numapphosts;
            else numcheck = numhosts;
            
            for (int i = 0; i < numcheck; i++) {
                token = st.nextToken();
                if (app.apphosts[i].equals(app.localhost)) {
                    host = token;
                    break;
                }
            }
            
            return(host);
        }
        catch (Exception e) {
            trace.error(appname + ":" + name + " - findDistHost() Exception: " + e.getMessage());
        }
        
        return(host);
    }
    
    public void assignPids(String pidstring) {
        trace.debug("Running in assignPids() - pidstring = " + pidstring);
        if (pidstring == null) return;
        if (type.equals("PROCESS_GROUP")) {
            StringTokenizer st = new StringTokenizer(pidstring);
            if (subcompnum != st.countTokens()) {
                trace.error("ERROR: wrong number of pids (" + pidstring + ")in message from " 
                                    + "Primary RedMan for component " + appname + ":" + name);
                return;
            }
            String stpid;
            for (int i=0; i < subcompnum; i++) {
                stpid = st.nextToken();
                trace.debug("Will assign pid " + stpid + " to process " 
                                  + appname + ":" + name + ":" + subcompmons[i].pname);
                subcompmons[i].subcomppid = Integer.parseInt(stpid);
            }
        }
        else {
            StringTokenizer st = new StringTokenizer(pidstring);
            pidstring = pidstring.trim();
            if ((st.countTokens()) != 1) {
                trace.error("ERROR: invalid pid (" + pidstring + ") in message from " 
                                    + "Primary RedMan for component " + appname + ":" + name);
                return;
            }
            trace.debug("Will assign pid " + pidstring + " to process " + appname + ":" + name);
            comppid = Integer.parseInt(pidstring);
        }
    }
    
    
    public void notifyLocalCompOfStateChange(String state) {
        if (type.equals("PROCESS_GROUP") && managed) 
            for (int i = 0; i < subcompnum; i++) subcompmons[i].notifySubCompOfStateChange(state);
        else if (managed) {
            if (out == null) {
                trace.debug("Output stream to proc " + appname + ":" + name 
                                + " is null. Can't notify of state change to " + state);
            }
            else {
                trace.debug("Will notify proc " + appname + ":" + name 
                                                + " of state change to " + state);
                out.println("CMD " + state);
                out.flush();
            }
        }
    }
    
    public void notifyLocalCompOfRemStateChange(String remstate) {
        if (type.equals("PROCESS_GROUP") && remeventmsg) {
            for (int i = 0; i < subcompnum; i++) {
                trace.debug("Will notify subcomponent " + appname + ":" 
                                      + name + ":" + subcompmons[i].pname 
                                      + " of rem state change to " + remstate);
                subcompmons[i].notifySubCompOfRemStateChange(remstate);
            }
        }
        else if (remeventmsg) {
            if (out == null) {
                trace.debug("Output stream to proc " + appname + ":" + name 
                                + " is null. Can't notify of rem state change to " + remstate);
            }
            else {
                trace.debug("Will notify proc " + appname + ":" 
                                                       + name + " of rem state change: " + remstate);
                out.println("EVENT REMOTE " + remstate);
                out.flush();
            }
        }
    }
    
    public void notifyLocalCompOfLocalEvent(String event) {
        if (event.startsWith("COMP_") && (event.indexOf(name) != -1)) {
            trace.debug("Local event '" + event + "' not sent to component " + name);
            return;
        }
        if (type.equals("PROCESS_GROUP")) {
            for (int i = 0; i < subcompnum; i++) {
                trace.debug("Will notify subcomponent " + appname + ":" 
                                      + name + ":" + subcompmons[i].pname 
                                      + " of local event " + event);
                subcompmons[i].notifySubCompOfLocalEvent(event);
            }
        }
        else if (managed) {
            if (out == null) {
                trace.debug("Output stream to proc " + appname + ":" + name 
                                + " is null. Can't notify of event '" + event + "'");
            }
            else {
                trace.debug("Will notify proc " + appname + ":" 
                                                       + name + " of local event " + event);
                out.println("EVENT LOCAL " + event);
                out.flush();
            }
        }
    }
 
    public void notifyLocalCompOfConfigChange(String change) {
        if (type.equals("PROCESS_GROUP")) {
            for (int i = 0; i < subcompnum; i++) {
                trace.debug("Will notify subcomponent " + appname + ":" 
                                      + name + ":" + subcompmons[i].pname 
                                      + " of config change " + change);
                subcompmons[i].notifySubCompOfConfigChange(change);
            }
        }
        else if (managed) {
            if (out == null) {
                trace.debug("Output stream to proc " + appname + ":" + name 
                                + " is null. Can't notify of config change '" + change + "'");
            }
            else {
                trace.debug("Will notify proc " + appname + ":" 
                                                       + name + " of config change " + change);
                String msg = "CONFIG_CHG ";
                if (change.equals("DESIGNATION")) {
                    msg += "DESIGNATION ";
                    if (app.designation) msg += "PRIMARY";
                    else msg += "SECONDARY";
                }
                if (change.equals("HA_MODEL")) {
                    msg += "HA_MODEL ";
                    if (app.hamodel.equals("1+1")) {
                        msg += "REDUNDANT " + app.remotehost;
                        if (app.designation) msg += " PRIMARY";
                        else msg += " SECONDARY";
                        if (app.failoverenabled) msg += " UNINHIBITED";
                        else msg += " INHIBITED";                       
                    }
                    else msg += "SIMPLEX";
                }
                out.println(msg);
                out.flush();
            }
        }
    }
        
    private boolean faultThresholdExceeded() {        
        if (threshold == 0 ) return(true);
        t2 = System.currentTimeMillis();
        numfault++;
        if (t1 == 0) {
            t1 = t2;            
            return(false);
        }
        else if ((t2 - t1) <= timewindow) {
                if (numfault > threshold) {
                    t1 = 0;
                    return(true);
                }
                else return(false);
             }
        else {
            numfault = 1;
            t1 = t2;
            return(false);
        }
    }   
    
    private void compProxy() {
    }
    
    public boolean findPid(int thepid) {
        String pidst = Integer.toString(thepid);        
        try {
            if (thepid >= 0) {
                Process p = Runtime.getRuntime().exec("/bin/ps -ef -o pid");
                //Process p = startProc("/bin/ps -ef -o pid");
                InputStream pin = p.getInputStream();
                InputStreamReader cin = new InputStreamReader(pin);
                BufferedReader in = new BufferedReader(cin);
            
                String linepid;
      
                do {
                    linepid = in.readLine();
                    if (linepid != null) {
                        linepid = linepid.trim();
                        if (linepid.equals(pidst)) {
                            trace.debug("Found pid '" + pidst + "' for " + appname + ":" + name);
                            return(true);
                        }
                    }
                }
                while (linepid != null);
            }
            else {
                trace.debug("Pid '" + pidst + "' for " + appname + ":" + name + " is invalid");
            }
        }
        catch (java.io.IOException e) {
             trace.error("Got exception trying to find pid '" + pidst + "' - " + e.getMessage());
        }
        trace.debug("Pid '" + pidst + "' for " + appname + ":" + name + " not found");        
        return(false);
    }    
    
    public static int findProcess(String proc) {
        boolean noproc = true;
        int val = -1;        
        try {
            if (proc != null) {
                Process p = Runtime.getRuntime().exec("/bin/ps -ef -o args,pid");
                //Process p = startProc("/bin/ps -ef -o args,pid");
                InputStream pin = p.getInputStream();
                InputStreamReader cin = new InputStreamReader(pin);
                BufferedReader in = new BufferedReader(cin);
            
                String line, pid, comm, ppid, token;
      
                do {
                    line = in.readLine();
                    if (line != null) {
                        StringTokenizer st = new StringTokenizer(line);                       
                        int numtokens = st.countTokens();
                        comm = st.nextToken();
                        int i;
                        if (numtokens > 2) {
                            for (i=numtokens; i > 2; i--) {
                                comm += " " + st.nextToken();
                            }
                        }                           
                        // trace.debug(appname + ":" + name + ": Command = " + comm);
                        if (comm.indexOf(proc) >= 0) {
                            pid = st.nextToken();
                            val = Integer.parseInt(pid);
                            break;
                        }
                    }
                }
                while (line != null);
            }
        }
        catch (java.io.IOException e) {
             trace.error("findProcess() IOException: " + e.getMessage());
             System.exit(3);
        }
        if (val < 0) {
            trace.error("CompMon - Process " + proc + " not found");
        }
        
        return(val);
    }
    
    public static int[] findProcesses(String proc) {
        boolean noproc = true;
        int[] pids = new int[5];
        int numpids = 0;
        int val = -1;
                
        try {
            if (proc != null) {
                Process p = Runtime.getRuntime().exec("/bin/ps -ef -o args,pid");
                //Process p = startProc("/bin/ps -ef -o args,pid");
                InputStream pin = p.getInputStream();
                InputStreamReader cin = new InputStreamReader(pin);
                BufferedReader in = new BufferedReader(cin);
            
                String line, pid, comm, ppid, token;      
                do {
                    line = in.readLine();
                    if (line != null) {
                        StringTokenizer st = new StringTokenizer(line);                       
                        int numtokens = st.countTokens();
                        comm = st.nextToken();
                        int i;
                        if (numtokens > 2) {
                            for (i=numtokens; i > 2; i--) {
                                comm += " " + st.nextToken();
                            }
                        }                           
                        
                        if (comm.indexOf(proc) >= 0) {
                            trace.debug("CompMon - found process \"" + comm + "\"");
                            pid = st.nextToken();
                            val = Integer.parseInt(pid);
                            noproc = false;
                            pids[numpids] = val;
                            numpids++;
                        }
                    }
                }
                while (line != null);
            }
        }
        catch (java.io.IOException e) {
             trace.error("findProcesses() IOException: " + e.getMessage());
             System.exit(3);
        }
        if (noproc) {
            trace.error("CompMon - Process " + proc + " not found");
            return(null);
        }
        else {
            int[] foundpids = new int[numpids+1];
            foundpids[0] = numpids;
            for (int i=1; i <= numpids; i++) foundpids[i] = pids[i];
            return(foundpids);
        }
    }
    
    private void waitOnPid(String pname) {
        int pid;
        String thename;
        if (name.equals(pname)) thename = appname + ":" + name;
        else thename = appname + ":" + name + ":" + pname;
        try {
            pid = findProcess(pname);
            if (pid < 0) {
                trace.info("Pid for process " + thename + " no found. Returning ...");
                return;
            }
            trace.debug(appname + ":" + name + ": Starting pwait ...");
        
            Process p = Runtime.getRuntime().exec("/usr/proc/bin/pwait -v " + String.valueOf(pid));
            //Process p = startProc("/usr/proc/bin/pwait -v " + String.valueOf(pid));
            InputStream pin = p.getInputStream();
            InputStreamReader cin = new InputStreamReader(pin);
            BufferedReader in = new BufferedReader(cin);
            String cmdoutput = in.readLine();
            in.close();
        
            // if (cmdoutput != null) trace.debug(appname + ":" + name + " " + cmdoutput);
        }
        catch (IOException e) {
            trace.error("Exception running waintOnPid(" + thename + "): " + e.getMessage());
        }
    }
    
    private void waitOnPid(int pid) {
        try {
            Process p = Runtime.getRuntime().exec("/home/artur/workspace/RedMan/bin/redman/pwait " + String.valueOf(pid));
            //Process p = startProc("/usr/proc/bin/pwait -v " + String.valueOf(pid));
            InputStream pin = p.getInputStream();
            InputStreamReader cin = new InputStreamReader(pin);
            BufferedReader in = new BufferedReader(cin);
            String cmdoutput = in.readLine();
            in.close();
        
            if (cmdoutput != null) trace.debug(appname + ":" + name 
                                                       + " - Output of 'pwait " + String.valueOf(pid)
                                                       + "' = [ " + cmdoutput + " ]");
        }
        catch (IOException e) {
            trace.error("Exception running waitOnPid(" + String.valueOf(pid) + "):" + e.getMessage() 
                                                       + " - Killing process pid " + String.valueOf(pid));
            killPid(pid);
        }
    }
    
    static void renicePid(String procname, int pid) {
        if (!RedMan.username.equals("root")) {
            trace.debug("Could not renice process " + procname + "(Pid=" + String.valueOf(pid) 
                                    + ") - RedMan user = " + RedMan.username + " (not root)");
            return;
        }
        try {
            Runtime.getRuntime().exec("/bin/renice 0 " + String.valueOf(pid));
            trace.debug("Process " + procname + "(Pid=" 
                                   + String.valueOf(pid) + ") was reniced to 0");
        }
        catch (IOException e) {
            trace.warn("Exception running renice on process " + procname 
                                + "(Pid=" + String.valueOf(pid) + ") - " + e.getMessage());
        }
    }
    
    public void startComponent() {
        newlystarted = true;
        if (type.equals("PROCESS_GROUP")) startProcessGroup();
        else startUniprocessComponent();
    }
    
    private Process startProc(String proc) throws IOException {
        Process p;
        String cmd;
        //String cmd = "/bin/priocntl -e -c TS su xteraems " + proc;
        if (RedMan.username.equals("root") && !app.username.equals("root")) 
            cmd = "su " + app.username + " " + proc;
        else cmd = proc;
        trace.debug("Will execute cmd '" + cmd + "'");

        p = Runtime.getRuntime().exec(cmd);
        return(p);
    }
    
    private void startUniprocessComponent() {
       String stpid;
       try {
            out = null;
            trace.info("\nStarting process " + appname 
                                        + ":" + name + " (\"" + startpath + "\")");                 
            //p = Runtime.getRuntime().exec(startpath);
            p = startProc(startpath);
            compdead = false;
            
            trace.info("Started process " + appname + ":" + name + " - Will get pid");
                
            // Obtain process pid from starting script
            InputStream pin = p.getInputStream();
            InputStreamReader cin = new InputStreamReader(pin);
            BufferedReader in = new BufferedReader(cin);
            stpid = in.readLine(); 
            String procname = appname + ":" + name;
            if (stpid != null) {                
                int pid = Integer.parseInt(stpid);
                renicePid(procname, pid);
                trace.info("Got pid from process " + procname + ", PID = " + stpid);
                in.close();
                comppid = pid;
            }
            else trace.error("Pid from process " + procname + " was null");
       }
       catch (Exception e) {
           trace.error(appname + ":" + name + " - startUniprocessComponent() Exception: " + e.getMessage());
           return;
       }
       
       if (stpid != null) sendPidStringToBackupRedMan(stpid);
    }
    
    private void verifyCompRegistration() {
        trace.debug("Task regverifier running for process " + appname + ":" + name);
        if (out == null) {
            trace.error("Managed process " + appname + ":" 
                                           + name + " did not register - Will kill it");
            killPid(comppid);
        }
        else waitToSpawnUniprocessCompTestThreads();
    }
    
    public void waitToSpawnUniprocessCompTestThreads() {
        String state;
        int counter = 0;
        boolean spawn = true;
        
        // Check if there are test threads already running for this uniprocess component
        // If that is true, terminate them. This takes care of the scenario in which an
        // already running component looses its connection with the RedMan and reconnects.
        if (tests != null) terminateUniprocessCompTestThreads();
        
        while (true) {
            state = app.getSMAState();
            if (state.equals("ACTIVE") || state.equals("STANDBY") 
                                       || state.equals("MAINTENANCE")) break; 
            else if (counter > app.statechgtime) {
                      spawn = false;
                      break;
            }
            trace.debug("Task will sleep 3 secs before calling spawnUniprocessCompTestThreads()");
            try {
                counter += 3000;
                sleep(3000);
            }
            catch (InterruptedException e) {
                trace.warn("regverifier task sleep interrupted");
            }
        }
        if (spawn) spawnUniprocessCompTestThreads();
    }
    
    private void monitorProcess() { 
       TimerTask regverifier = null;
       int watchedpid = comppid;
       newlystarted = false;
       try {
           if (managed) {
               // Schedule task that will start one or more components
               regverifier = new TimerTask() {
                   public void run() {  verifyCompRegistration(); }
               };
               Timer timer = new Timer();
               timer.schedule(regverifier, app.inittime);
           }
           else spawnUniprocessCompTestThreads(); 
           trace.debug("Will wait on pid " + Integer.toString(watchedpid) 
                                           + " for process " + appname + ":" + name);                   
           waitOnPid(watchedpid);
           if (regverifier != null) regverifier.cancel();
           trace.error("Process " + appname + ":" + name + " (PID =" 
                                  + Integer.toString(watchedpid) + ") died - comppid="
                                  + Integer.toString(comppid) + ", newlystarted=" + newlystarted);
           if ((!newlystarted) && (comppid == watchedpid) && (comppid != -1)) {
               cleanupUniprocessComponent(true);
               compdead = true;
               comppid = -1;
               out = null;               
               killDependentComponents();
           }
       }
       catch (Exception e) {
           trace.error(appname + ":" + name + " - monitorProcess() Exception: " + e.getMessage());
       }
    }
    
    private void killDependentComponents() {
        if (dependentcomps != null) {
            StringTokenizer st = new StringTokenizer(dependentcomps);
            String comp;
            int index, i = 0;
            while (st.hasMoreTokens()) {
                comp = st.nextToken();
                index = app.findCompObjIndex(comp);
                if (index >= 0) {
                    i++;
                    trace.debug(appname + ":" + name 
                        + " - will cleanup (kill) dependent component '" + comp + "'");
                    app.compmons[index].cleanupComponent(true);
                }
            }
            trace.debug(appname + ":" + name + " - cleaned up (killed) " + String.valueOf(i)    
                                + " dependent components");
        }
        else {
            trace.debug(appname + ":" + name + " - no dependent components found");
        }                    
    }
    
    void spawnUniprocessCompTestThreads() {
        String tname, testnames;
        String objname = appname + "_" + name;
        
        testnames = app.appstatictable.getProperty(appname + "_" + name + "_TESTS");
        if (testnames == null) return;
        StringTokenizer st = new StringTokenizer(testnames);
        numtests = st.countTokens();
        trace.debug(appname + "_" + name + " has " + numtests + " defined tests");
        
        tests = new TestThread[numtests];
        for (int i = 0; i < numtests; i++) {
            tname = st.nextToken();
            tests[i] = new TestThread(objname, tname, app, this, null); 
        }
    }
    
    private void terminateUniprocessCompTestThreads() {
        if (numtests > 0) {
            trace.info("Component " + name + " has " + numtests + " test threads - Will terminate them");
            for (int i = 0; i < numtests; i++) 
                if (tests[i] != null) tests[i].terminate();
        }
        tests = null;
        numtests = 0;
    }
    
    private void terminateProcessGroupTestThreads() {
        for (int i = 0; i < subcompnum; i++) {
            if (subcompmons[i] != null) subcompmons[i].terminateSubCompTestThreads();
        }
    }
               
    public static void killProcess(String proc) {
        int pid;
        try {
            pid = findProcess(proc);
            if (pid < 0) {
                trace.error("RedMan tried to kill process " 
                                                + proc + ", but could not find it");
                return;
            }
            else {
                trace.debug("RedMan will forcefully kill process "
                                            + proc + " with pid = " + Integer.toString(pid));
                killPid(pid);
            }
        }
        catch (Exception e) {
            trace.error("killProcess() Exception: " + e.getMessage());
        }
    }

    public int getProcPid(String procname) {
        int pid = -1;
        if (type.equals("PROCESS_GROUP")) {
            for (int i = 0; i < subcompnum; i++) {
                pid = subcompmons[i].getProcPid(procname);
                if (pid != -1) break;
            }
        }
        else if (procname.equals(name)) pid = comppid;
        return(pid);
    }
    
    public static void killPid(int pid) {
        try {
            if (pid < 0) return;
            trace.debug("CompMon - Will kill process with pid = " + pid);
        
            Process p = Runtime.getRuntime().exec("/bin/kill -9 " + String.valueOf(pid));
            //Process p = startProc("/bin/kill -9 " + String.valueOf(pid));
            InputStream pin = p.getInputStream();
            InputStreamReader cin = new InputStreamReader(pin);
            BufferedReader in = new BufferedReader(cin);
            String cmdoutput = in.readLine();
            in.close();
        
            if (cmdoutput != null) trace.debug("Output from 'kill -9' on pid(" 
                            + String.valueOf(pid) + "): " + cmdoutput);
        }
        catch (IOException e) {
            trace.error("killPid() IOException: " + e.getMessage());
        }
    }
    
    private void looseEndsCleaningUniprocessComponent(boolean killdependents) {
        compdead = true;
        comppid = -1;
        out = null;               
        if (killdependents) killDependentComponents();        
    }
    
    private void cleanupUniprocessComponent(boolean kill) {
        trace.debug(appname + ":" + name + " shutdownpath = " + shutdownpath);
        trace.debug(appname + ":" + name + " killpath = " + killpath);
        terminateUniprocessCompTestThreads();
        if ((shutdownpath != null) && (!kill)) {
            try {
                String cmd = shutdownpath + " " + String.valueOf(comppid);
                trace.error("Shutting down uniprocess component " + appname 
                                        + ":" + name + " (\"" + cmd + "\")"); 
                Runtime.getRuntime().exec(cmd);
                //startProc(cmd);
            }
            catch (Exception e) {
                trace.error("Could not execute shutdown script '" + shutdownpath 
                                          + "' for uniprocess component " 
                                          + appname + ":" + name + ", PID = " + String.valueOf(comppid)
                                          + " - Exception: " + e.getMessage());
                // Kill process
                if (killpath != null) {
                    String cmd = killpath + " " + String.valueOf(comppid);
                    trace.error("Killing uniprocess component " + appname 
                                        + ":" + name + " and its children (\"" + cmd + "\")"); 
                    try {
                        Runtime.getRuntime().exec(cmd);
                        //startProc(cmd);
                    }
                    catch (IOException e1) {
                        trace.error("Could not execute the kill script '" + killpath 
                                     + "' for process " + appname + ":" + name + ", PID = " 
                                     + String.valueOf(comppid) + " - Exception: " + e1.getMessage());
                        killPid(comppid);
                    }
                }
                else {
                    trace.error("Killing uniprocess component " + appname + ":" 
                                                               + name + " (\"killPid()\")");
                    killPid(comppid);
                }
                looseEndsCleaningUniprocessComponent(true);
            }
        }
        else {
            // Kill process
            if (killpath != null) {
            	String cmd = "killall " + String.valueOf(comppid);
            	//String cmd = killpath + " " + String.valueOf(comppid);
                if (comppid != -1) 
                    trace.error("Killing uniprocess component " + appname 
                                        + ":" + name + " and its children (\"" + cmd + "\")");
                else
                    trace.error("Killing all children of uniprocess component " + appname 
                                        + ":" + name + " (\"" + cmd + "\")");
                try {
                    Runtime.getRuntime().exec(cmd);
                    //startProc(cmd);
                }
                catch (IOException e2) {
                    trace.error("Could not execute the kill script '" + killpath 
                                     + "' for uniprocess component " + appname + ":" + name + ", PID = " 
                                     + String.valueOf(comppid) + " - Exception: " + e2.getMessage());
                    killPid(comppid);
                }
            }
            else killPid(comppid);
            looseEndsCleaningUniprocessComponent(true);
        }
    }
    
    public void killProcessGroup() {
        for (int i = 0; i < subcompnum; i++) {
            if (!subcompmons[i].grprocdead) {
                // killProcess(subcompmons[i].pname);
                killPid(subcompmons[i].subcomppid);
            }
        }
    }

    private void looseEndsCleaningProcessGroup(boolean killdependents) {
        compdead = true;
        comppid = -1;
        out = null;          
        for (int i=0; i < subcompnum; i++) {
            subcompmons[i].subcomppid = -1;
            subcompmons[i].out = null;
            subcompmons[i].grprocdead = true;
        }
        if (killdependents) killDependentComponents();        
    }
    
    public void cleanupProcessGroup(boolean kill) {
        // Observation: no kill script is executed for a process group
        terminateProcessGroupTestThreads();
        if ((shutdownpath != null) && (!kill)) {
            try {
                String cmd = shutdownpath;
                for (int i=0; i < subcompnum; i++) {
                    cmd += " " + String.valueOf(subcompmons[i].subcomppid);
                }
                trace.info("Shutting down process group" + appname 
                                        + ":" + name + " (\"" + cmd + "\")");                 
                Runtime.getRuntime().exec(cmd);
                //startProc(cmd);
            }
            catch (IOException e) {
                trace.error("Exception:" + e.getMessage() 
                                         + " - Will forcefully kill process group " + name);
                killProcessGroup();
                looseEndsCleaningProcessGroup(true);
            }
        }
        else {
            trace.info("Will forcefully kill process group " + appname + ":" + name);
            killProcessGroup();
            looseEndsCleaningProcessGroup(true);
        }
    }
    
    public void looseEndsCleaning(boolean killdependents) {
        if (type.equals("PROCESS_GROUP")) looseEndsCleaningProcessGroup(killdependents);
        else looseEndsCleaningUniprocessComponent(killdependents);        
        trace.debug("looseEndsCleaning() executed for component " + name 
                               + " (killdependents = " + killdependents + ")");
    }
    
    public void cleanupComponent(boolean kill) {
        if (!compdead) {
            if (type.equals("PROCESS_GROUP")) cleanupProcessGroup(kill);
            else cleanupUniprocessComponent(kill);
            if (kill) compdead = true;
        }
        else trace.debug("Component " + name + " already dead - No need to clean it up");
    }
    
    public boolean componentIsDown() {
        if (type.equals("PROCESS_GROUP")) {
            for (int i = 0; i < subcompnum; i++) {
                if (findPid(comppid)) return(false);
            }
            return(true);
        }
        else {
            if (findPid(comppid)) return(false);
            else return(true);
        }
    }
    
    private void createProcessGroupObjects() {
        // Get subcomponent names
        String procids = app.appstatictable.getProperty(appname + "_" 
                                                    + name + "_" + "SUBCOMPONENTS");
        StringTokenizer st = new StringTokenizer(procids);
        subcompnum = st.countTokens();
        subcompmons = new SubCompMon[subcompnum];
        
        // Create SubCompMon objects
        int i = 0;
        while (st.hasMoreTokens()) {
            String procid = st.nextToken();
            String procname = app.appstatictable.getProperty(appname + "_" 
                                                     + name + "_" + procid);
            procname = procname.trim();
            subcompmons[i] = new SubCompMon(procname, this);         
            i++;
        }        
    }

    public void sendPidStringToBackupRedMan(String pidstring) {
        // Propagate the pid string to the backup RedMan
        if (RedMan.backupwriter != null) {
            RedMan.backupwriter.println(appname + " PID " + name + " " + pidstring);
            RedMan.backupwriter.flush();
            trace.debug("PRIMARY RedMan sent msg to BACKUP RedMan: \"" 
                                 + appname + " PID " + name + " " + pidstring + "\"");
        }
        else {
            trace.warn("PRIMARY RedMan could not send msg to BACKUP RedMan: \""
                         + appname + " PID " + name + " " + pidstring + "\" - No output stream");
        }
    }
    
    private void startProcessGroup() {
        BufferedReader in = null;
        String stpid = null, pidstring = " ";
        out = null;
        compdead = false;
        try {
            trace.info("Starting process group " + appname + ":" + name 
                                                + " (\"" + startpath + "\")" ); 
            // Executing process group script
            //p = Runtime.getRuntime().exec(startpath);       
            p = startProc(startpath);
                
            // Obtain input stream from starting script
            InputStream pin = p.getInputStream();
            InputStreamReader cin = new InputStreamReader(pin);
            in = new BufferedReader(cin);
                             
            // Get pids for process group processes
            for (int i=0; i < subcompnum; i++) {                              
                stpid = in.readLine();
                if (stpid != null) {
                    stpid = stpid.trim();
                    int pid = Integer.parseInt(stpid);
                    pidstring += stpid + " ";                
                    String procname = appname + ":" + name + ":" + subcompmons[i].pname;
                    trace.info("Got pid for process " + procname + ", PID = " + stpid);
                    renicePid(procname, pid);
                    subcompmons[i].subcomppid = pid;
                }
            }
            comppid = 0;
            out = null;
            in.close();
            
            if (stpid != null) sendPidStringToBackupRedMan(pidstring);
       }
       catch (Exception e) {
           trace.error(appname + ":" + name + " - startProcessGroup() Exception: " + e.getMessage());
       }
    }
    
    public void sendCompPidsToBackupRedMan() {
        String pidstring = " ";
        if (type.equals("PROCESS_GROUP")) {
            for (int i=0; i < subcompnum; i++) {                              
                pidstring += String.valueOf(subcompmons[i].subcomppid) + " ";          
            }
        }
        else pidstring += String.valueOf(comppid);
        trace.info("Will try to sent pid string '" + pidstring + "' for component " 
                                    + appname + ":" + name + " to the backup RedMan");
        sendPidStringToBackupRedMan(pidstring);
    }
        
    
    private void monitorProcessGroup() {
        int i = 0;
        String stpid;
        newlystarted = false;
        try {            
            // Start threads to wait on the processes of the process group
            for (i = 0; i < subcompnum; i++) {
                // trace.debug("Will start thread to wait on group process " 
                //                    + appname + ":" + name + ":" + subcompmons[i].pname);
                subcompmons[i].start();
            }
                
            try {
                synchronized (pgroup_sem) {
                    pgroup_sem.wait();
                }
            }
            catch (InterruptedException e) {
                trace.debug(appname + ":" + name + " - pgrpup_sem.wait() interrupted");
            }
            
            boolean cleanup = false;
            for ( i = 0; i < subcompnum; i++) {
                if (subcompmons[i].oldinstance) cleanup = true;
                subcompmons[i].oldinstance = false;
            }
                
            if (cleanup) cleanupProcessGroup(true);
        }
        catch (Exception e) {
            trace.debug(appname + ":" + name + " - monitorProcessGroup() Exception: " + e.getMessage());
        }
    }
    
    private void waitForNotificationToMonitorAgain() {
        try {
            synchronized (app.monitor_sem) {
                trace.debug("Monitoring thread for component " 
                                    + appname + ":" + name + " will wait on semaphore");
                app.monitor_sem.wait();
                trace.debug("Monitoring thread for component " 
                                    + appname + ":" + name + " got semaphore notification");
            }
        }
        catch (InterruptedException e) {
            trace.warn(appname + ":" + name + " - app.monitor_sem.wait() interrupted");
        }        
    }
    
    public void run() {    // CompMon
        boolean start = false;
        trace.debug("COMPONENT " + appname + ":" + name);
        while (true) {           
            try {                
                if (type.equals("PROCESS_GROUP")) {
                    if (start) {
                        startProcessGroup();
                        app.notifyAppOfLocalEvent("COMP_START " + name);
                        start = false;
                    }
                    monitorProcessGroup();
                }
                else {
                    if (start) {
                        startUniprocessComponent();
                        app.notifyAppOfLocalEvent("COMP_START " + name);
                        start = false;
                    }
                    monitorProcess();
                }
                app.notifyAppOfLocalEvent("COMP_DEATH " + name);
                if (failureHandled) {
                    trace.debug("Component " + name + " died, but failure/shutdown of app " 
                                                    + appname + " has already been handled");
                    waitForNotificationToMonitorAgain();
                    failureHandled = false;
                }
                else {
                    if (faultThresholdExceeded()) {
                        trace.info("Fault threshold exceeded for component " + appname + ":" + name);
                        compdead = true;
                        handlingfailure = true;
                        app.handleFailure("FLT");
                        handlingfailure = false;
                        waitForNotificationToMonitorAgain();
                        failureHandled = false;
                    }                
                    else start = true;
                    firsttime = false;
                }
            }
            catch (Exception e) {
                trace.debug(appname + ":" + name + " - CompMon.run() Exception: " + e.getMessage());
            }
            trace.debug("Monitoring thread for component " + appname + ":" + name 
                                           + " will loop around to monitor again");
        }
    }
    
    public class SubCompMon extends Thread {
        String pname;
        boolean grprocdead = false;
        boolean oldinstance = false;
        Socket so = null;
        boolean submanaged = false;
        boolean subremeventmsg = false;
        boolean subloceventmsg = false;
        PrintWriter out = null;
        TestThread[] tests = null;
        int numtests = 0;
        int subcomppid = -1;
        CompMon comp = null;
        
        public int getProcPid(String procname) {
            if (procname.equals(pname)) return(subcomppid);
            else return(-1);
        }
        
        public SubCompMon(String thename, CompMon comp) {
            this.comp = comp;
            pname = thename;
            if (msgsspecified) {
                submanaged = managed;
                subremeventmsg = remeventmsg;
                subloceventmsg = loceventmsg;
            }
            else {
                String stmsgs = 
                    app.appstatictable.getProperty(appname + "_" + name + "_" + pname + "_MESSAGES");
                if (stmsgs != null) {
                    submanaged = true;
                    String msg;
                    StringTokenizer st = new StringTokenizer(stmsgs);
                    while (st.hasMoreTokens()) {
                        msg = st.nextToken();
                        if (msg.equals("LOCAL_EVENT")) subloceventmsg = true;
                        if (msg.equals("REMOTE_EVENT")) subremeventmsg = true;
                    }
                }
            }
            if (submanaged) {
                if (firsttime) app.nummanaged++;
                managed = true;
            }
            if (subremeventmsg) remeventmsg = true;
            if (subloceventmsg) loceventmsg = true;
        }
        
        public void notifySubCompOfStateChange(String state) {
            if (submanaged) {
                if (out == null) {
                    trace.debug("Output stream to proc " 
                                        + appname + ":" + name + ":" + pname 
                                        + " is null. Can't send state change cmd: '" + state + "'");
                }
                else {
                    out.println("CMD " + state);
                    out.flush();
                }
            }
        }
        
        public void notifySubCompOfRemStateChange(String remstate) {
            if (subremeventmsg) {
                if (out == null) {
                    trace.debug("Output stream to proc " 
                                        + appname + ":" + name + ":" + pname 
                                        + " is null. Can't send remote event: '" + remstate + "'");
                }
                else {
                    out.println("EVENT REMOTE " + remstate);
                    out.flush();
                }
            }
        }
        
        public void notifySubCompOfLocalEvent(String event) {
            if (subloceventmsg) {
                if (out == null) {
                    trace.debug("Output stream to proc " 
                                        + appname + ":" + name + ":" + pname 
                                        + " is null. Can't send local event: '" + event + "'");
                }
                else {
                    out.println("EVENT LOCAL " + event);
                    out.flush();
                }
            }
        }
        
        public void notifySubCompOfConfigChange(String change) {
            if (submanaged) {
                if (out == null) {
                    trace.debug("Output stream to proc " + appname + ":" + name  + ":" + pname
                                    + " is null. Can't notify of config change '" + change + "'");
                }
                else {
                    trace.debug("Will notify proc " + appname + ":" + name + ":" 
                                                       + pname + " of config change " + change);
                    String msg = "CONFIG_CHG ";
                    if (change.equals("DESIGNATION")) {
                        msg += "DESIGNATION ";
                        if (app.designation) msg += "PRIMARY";
                        else msg += "SECONDARY";
                    }
                    if (change.equals("HA_MODEL")) {
                        msg += "HA_MODEL ";
                        if (app.hamodel.equals("1+1")) {
                            msg += "REDUNDANT " + app.remotehost;
                            if (app.designation) msg += " PRIMARY";
                            else msg += " SECONDARY";
                            if (app.failoverenabled) msg += " UNINHIBITED";
                            else msg += " INHIBITED";                       
                        }
                        else msg += "SIMPLEX";
                    }
                    out.println(msg);
                    out.flush();
                }
            }
        }
        
        void spawnSubCompTestThreads() {
            String tname, testnames;
            String objname = appname + "_" + name + "_" + pname;
            
            testnames = app.appstatictable.getProperty( objname + "_TESTS");
            if (testnames == null) return;
         
            StringTokenizer st = new StringTokenizer(testnames);
            numtests = st.countTokens();
            
            tests = new TestThread[numtests];
            for (int i = 0; i < numtests; i++) {
                tname = st.nextToken();
                tests[i] = new TestThread(objname, tname, app, comp, this); 
            }
        } 
        
        void terminateSubCompTestThreads() {
            if (numtests > 0) {
                for (int i = 0; i < numtests; i++) 
                    if (tests[i] != null) tests[i].terminate();
            }
            tests = null;
            numtests = 0;
        }
        
        private void verifySubCompRegistration() {
            trace.debug("Task regverifier running for process " 
                                      + appname + ":" + name + ":" + pname);
            if (out == null) {
                trace.error("Managed process " + appname + ":" 
                                               + name + " did not register - Will kill it");
                killPid(comppid);
            }
            else waitToSpawnSubCompTestThreads();
        }
        
        public void waitToSpawnSubCompTestThreads() {    
            String state;
            boolean spawn = true;
            int counter = 0;

            // Check if there are test threads running for this component
            // and terminate them if that is true. This takes care of the 
            // scenario in which an already running component looses its 
            // connection with the RedMan and reconnects.
            if (tests != null) terminateSubCompTestThreads();
            
            while (true) {
                state = app.getSMAState();
                if (state.equals("ACTIVE") || state.equals("STANDBY") 
                                           || state.equals("MAINTENANCE")) break; 
                else if (counter > app.statechgtime) {
                         spawn = false;
                         break;
                }
                trace.debug("Task will sleep 3 secs before calling spawnSubCompTestThreads()");
                try {
                    counter += 3000;
                    sleep(3000);
                }
                catch (InterruptedException e) {
                    trace.warn("regverifier task sleep interrupted");
                }
            }
            if (spawn) spawnSubCompTestThreads();   
        }        

        private void monitorSubComponent() {
            TimerTask regverifier = null;
            int watchedpid = subcomppid;
            if (submanaged) {
                // Schedule task that will start one or more components
                regverifier = new TimerTask() {
                    public void run() {  verifySubCompRegistration(); }
                };
                Timer timer = new Timer();
                timer.schedule(regverifier, app.inittime);
            }
            else spawnSubCompTestThreads(); 
           
            trace.debug("Will wait on pid for subcomponent " + appname + ":" + name 
                                        + ":" + pname + ", PID = " + String.valueOf(watchedpid));
            waitOnPid(watchedpid);
            trace.debug("Subcomponent " + appname + ":" + name + ":" + pname 
                                        + ", PID = " + String.valueOf(watchedpid) + ", died");
            if (regverifier != null) regverifier.cancel();
            
            try {
                synchronized (pgroup_sem) {
                    pgroup_sem.notify();
                }
            }
            catch (Exception e) {
                trace.debug(appname + ":" + name + ":" + pname 
                                    + " - monitorSubComponent() Exception:" + e.getMessage());
            }
        
            if ((!newlystarted) && (subcomppid == watchedpid) && (subcomppid != -1)) {
                oldinstance = true;
            }
        }
                                          
        public void run() {
            monitorSubComponent();
        }
    }
}